import dataBase 
from mainLib import *

def spider(db,hashdb,savepage = False,reproduce = False):
    url = db.pop()
    try:
        result = download(url = url)
        if result:
            urls = analyse(result)
            content = pick(result)
            saveContent(content = content)
            if savepage:
                saveHtml(result)
            if urls and reproduce:
                saveUrl(db,hashdb,urls)
        else:
            pass
    except Exception as e:
        print(e)